#!/usr/bin/env python

import wsgiref.handlers
from google.appengine.ext import db
from google.appengine.ext import webapp
from google.appengine.api.labs import taskqueue
from schema import Document, Fetch, checksum_from_key_name

import logging

class DocumentFinder(webapp.RequestHandler):
  def post(self):
    PAGESIZE = 5
    next = None
    document_key = self.request.get("document_key")
    
    documents = Document.all().order("__key__")
    if document_key:
      logging.debug("This is a continuation, starting with %s." % document_key)
      next_documents = documents.filter('__key__ >=', db.Key(document_key))
      batch_of_documents = next_documents.fetch(PAGESIZE+1)
    else:
      logging.debug("This is the first request.")
      batch_of_documents = documents.fetch(PAGESIZE+1)

    if len(batch_of_documents) == PAGESIZE+1:
      logging.debug("There are enough documents to make another batch.")
      next = batch_of_documents[-1]
      logging.debug("The next one should start from %s" % str(next.key()))
      task = taskqueue.Task(
        url='/find-documents-to-fetch',
        params={
          'document_key': str(next.key()),
        }
      )
      task.add(queue_name='fetch')
      batch_of_documents = batch_of_documents[:PAGESIZE]

    logging.debug("This batch has %i documents." % len(batch_of_documents))

    for document in batch_of_documents:
      docid = db.ByteString(checksum_from_key_name(document.key().name()))
      fetch = Fetch.all().filter('docid =', docid).get()
      logging.debug("Has document %s been fetched before?" % (document.key()))
      if fetch is None:
        logging.debug("Creating task to fetch %s." % str(document.key()))
        task = taskqueue.Task(
          url='/fetch',
          params={
            'document_key': str(document.key()),
          }
        )
        task.add(queue_name='fetch')

def main():
  logging.getLogger().setLevel(logging.DEBUG)
  wsgiref.handlers.CGIHandler().run(webapp.WSGIApplication([
    ('/find-documents-to-fetch', DocumentFinder),
  ]))


if __name__ == '__main__':
  main()
